{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Evidently Tests "
   ],
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "attachments": {},
   "id": "2b27f18a877978e4"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "try:\n",
    "    import evidently\n",
    "except:\n",
    "    !pip install git+https://github.com/evidentlyai/evidently.git"
   ],
   "outputs": [],
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "id": "955fc0654c38b963"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "import numpy as np\n",
    "\n",
    "from sklearn import datasets\n",
    "from sklearn import ensemble\n",
    "\n",
    "from evidently.options import ColorOptions\n",
    "from evidently.test_suite import TestSuite\n",
    "from evidently.tests import *"
   ],
   "outputs": [],
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "id": "b0bc08717518f841"
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Prepare Datasets"
   ],
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "attachments": {},
   "id": "ab504f9437a3cbaf"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#Dataset for Data Quality and Integrity\n",
    "adult_data = datasets.fetch_openml(name='adult', version=2, as_frame=True)\n",
    "adult = adult_data.frame\n",
    "\n",
    "adult_ref = adult[~adult.education.isin(['Some-college', 'HS-grad', 'Bachelors'])]\n",
    "adult_cur = adult[adult.education.isin(['Some-college', 'HS-grad', 'Bachelors'])]\n",
    "\n",
    "adult_cur.iloc[:2000, 3:5] = np.nan"
   ],
   "outputs": [],
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "id": "68cd09f0b60fe5ec"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#Dataset for Regression\n",
    "housing_data = datasets.fetch_california_housing(as_frame=True)\n",
    "housing = housing_data.frame\n",
    "\n",
    "housing.rename(columns={'MedHouseVal': 'target'}, inplace=True)\n",
    "housing['prediction'] = housing_data['target'].values + np.random.normal(0, 3, housing.shape[0])\n",
    "\n",
    "housing_ref = housing.sample(n=5000, replace=False)\n",
    "housing_cur = housing.sample(n=5000, replace=False)"
   ],
   "outputs": [],
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "id": "5ee02586c06f2838"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#Dataset for binary label and probabilistic classification\n",
    "bcancer_data = datasets.load_breast_cancer(as_frame=True)\n",
    "bcancer = bcancer_data.frame\n",
    "\n",
    "bcancer_ref = bcancer.sample(n=300, replace=False)\n",
    "bcancer_cur = bcancer.sample(n=200, replace=False)\n",
    "\n",
    "bcancer_label_ref = bcancer_ref.copy(deep=True)\n",
    "bcancer_label_cur = bcancer_cur.copy(deep=True)\n",
    "\n",
    "model = ensemble.RandomForestClassifier(random_state=1, n_estimators=10)\n",
    "model.fit(bcancer_ref[bcancer_data.feature_names.tolist()], bcancer_ref.target)\n",
    "\n",
    "bcancer_ref['prediction'] = model.predict_proba(bcancer_ref[bcancer_data.feature_names.tolist()])[:, 1]\n",
    "bcancer_cur['prediction'] = model.predict_proba(bcancer_cur[bcancer_data.feature_names.tolist()])[:, 1]\n",
    "\n",
    "bcancer_label_ref['prediction'] = model.predict(bcancer_label_ref[bcancer_data.feature_names.tolist()])\n",
    "bcancer_label_cur['prediction'] = model.predict(bcancer_label_cur[bcancer_data.feature_names.tolist()])"
   ],
   "outputs": [],
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "id": "874ef992eaa7ba50"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#Dataset for Multiclass Classifcation (labels)\n",
    "iris_data = datasets.load_iris(as_frame=True)\n",
    "iris = iris_data.frame\n",
    "\n",
    "iris_ref = iris.sample(n=75, replace=False)\n",
    "iris_cur = iris.sample(n=75, replace=False)\n",
    "\n",
    "model = ensemble.RandomForestClassifier(random_state=1, n_estimators=3)\n",
    "model.fit(iris_ref[iris_data.feature_names], iris_ref.target)\n",
    "\n",
    "iris_ref['prediction'] = model.predict(iris_ref[iris_data.feature_names])\n",
    "iris_cur['prediction'] = model.predict(iris_cur[iris_data.feature_names])"
   ],
   "outputs": [],
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "id": "268401b9d1d7ea4f"
  },
  {
   "cell_type": "markdown",
   "source": [
    "## How to run TestSuites?"
   ],
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "attachments": {},
   "id": "18c65e8c0f51d39e"
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Data Intergrity Tests"
   ],
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "attachments": {},
   "id": "ac793fa510a7070f"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#dataset-level tests\n",
    "data_integrity_dataset_tests = TestSuite(tests=[\n",
    "    TestNumberOfColumns(),\n",
    "    TestNumberOfRows(),\n",
    "    TestNumberOfMissingValues(),\n",
    "    TestShareOfMissingValues(),\n",
    "    TestNumberOfColumnsWithMissingValues(),\n",
    "    TestNumberOfRowsWithMissingValues(),\n",
    "    TestShareOfColumnsWithMissingValues(),\n",
    "    TestShareOfRowsWithMissingValues(),\n",
    "    TestNumberOfDifferentMissingValues(),\n",
    "    TestNumberOfConstantColumns(),\n",
    "    TestNumberOfEmptyRows(),\n",
    "    TestNumberOfEmptyColumns(),\n",
    "    TestNumberOfDuplicatedRows(),\n",
    "    TestNumberOfDuplicatedColumns(),\n",
    "    TestColumnsType(),\n",
    "    \n",
    "])\n",
    "\n",
    "data_integrity_dataset_tests.run(reference_data=adult_ref, current_data=adult_cur)\n",
    "data_integrity_dataset_tests"
   ],
   "outputs": [],
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": true
   },
   "id": "7d20a9a4eb7deb81"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#column-level tests\n",
    "data_integrity_column_tests = TestSuite(tests=[\n",
    "    TestColumnNumberOfMissingValues(column_name='education'),\n",
    "    TestColumnShareOfMissingValues(column_name='education'),\n",
    "    TestColumnNumberOfDifferentMissingValues(column_name='education'),\n",
    "    TestColumnAllConstantValues(column_name='education'),\n",
    "    TestColumnAllUniqueValues(column_name='education'),\n",
    "    TestColumnRegExp(column_name='education',reg_exp='^[0..9]'),\n",
    "    TestCategoryShare(column_name='education', category='Some-college', lt=0.5),\n",
    "    TestCategoryShare(column_name='age', category=27., lt=0.5)\n",
    "])\n",
    "\n",
    "data_integrity_column_tests.run(reference_data=adult_ref, current_data=adult_cur)\n",
    "data_integrity_column_tests"
   ],
   "outputs": [],
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "id": "41ead38f21cdc450"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#test in a JSON format\n",
    "data_integrity_dataset_tests.json()"
   ],
   "outputs": [],
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": false
   },
   "id": "ca668b1e3d0008dc"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#tests as a python object\n",
    "data_integrity_dataset_tests.as_dict()"
   ],
   "outputs": [],
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": true
   },
   "id": "3a0ebb1eb6935a04"
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Data Quality Tests"
   ],
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "attachments": {},
   "id": "1982a8d227a0bfd0"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#dataset-level tests\n",
    "data_quality_dataset_tests = TestSuite(tests=[\n",
    "    TestConflictTarget(),\n",
    "    TestConflictPrediction(),\n",
    "    TestTargetPredictionCorrelation(),\n",
    "    TestHighlyCorrelatedColumns(),\n",
    "    TestTargetFeaturesCorrelations(),\n",
    "    TestPredictionFeaturesCorrelations(),\n",
    "    TestCorrelationChanges(),\n",
    "])\n",
    "\n",
    "data_quality_dataset_tests.run(reference_data=iris_ref, current_data=iris_cur)\n",
    "data_quality_dataset_tests"
   ],
   "outputs": [],
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "id": "fff5ba0fcb407c6f"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#column-level tests\n",
    "data_quality_column_tests = TestSuite(tests=[\n",
    "    TestColumnValueMin(column_name='education-num'),\n",
    "    TestColumnValueMax(column_name='education-num'),\n",
    "    TestColumnValueMean(column_name='education-num'),\n",
    "    TestColumnValueMedian(column_name='education-num'),\n",
    "    TestColumnValueStd(column_name='education-num'),\n",
    "    TestNumberOfUniqueValues(column_name='education'),\n",
    "    TestUniqueValuesShare(column_name='education'),\n",
    "    TestMostCommonValueShare(column_name='education'),\n",
    "    TestMeanInNSigmas(column_name='education-num'),\n",
    "    TestValueRange(column_name='education-num'),\n",
    "    TestNumberOfOutRangeValues(column_name='education-num'),\n",
    "    TestShareOfOutRangeValues(column_name='education-num'),\n",
    "    TestValueList(column_name='education'),\n",
    "    TestNumberOfOutListValues(column_name='education'),\n",
    "    TestShareOfOutListValues(column_name='education'),\n",
    "    TestColumnQuantile(column_name='education-num', quantile=0.25),\n",
    "    TestShareOfOutListValues(column_name='education-num'),\n",
    "])\n",
    "\n",
    "data_quality_column_tests.run(reference_data=adult_ref, current_data=adult_cur)\n",
    "data_quality_column_tests"
   ],
   "outputs": [],
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "id": "bef23cbaef8a0c65"
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Data Drift Tests"
   ],
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "attachments": {},
   "id": "3ffbe8f095f164c9"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#dataset-level tests\n",
    "data_drift_dataset_tests = TestSuite(tests=[\n",
    "    TestNumberOfDriftedColumns(),\n",
    "    TestShareOfDriftedColumns(),\n",
    "])\n",
    "\n",
    "data_drift_dataset_tests.run(reference_data=adult_ref, current_data=adult_cur)\n",
    "data_drift_dataset_tests"
   ],
   "outputs": [],
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "id": "25a85149a0c647a"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#column-level tests\n",
    "data_drift_column_tests = TestSuite(tests=[\n",
    "    TestColumnDrift(column_name='education-num', stattest='psi', stattest_threshold=0.3)\n",
    "])\n",
    "\n",
    "data_drift_column_tests.run(reference_data=adult_ref, current_data=adult_cur)\n",
    "data_drift_column_tests"
   ],
   "outputs": [],
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "id": "1d670bf1b214cc74"
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Regression Performance Tests"
   ],
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "attachments": {},
   "id": "93fcbee91eb91762"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#dataset-level tests\n",
    "regression_performance_dataset_tests = TestSuite(tests=[\n",
    "    TestValueMAE(),\n",
    "    TestValueRMSE(),\n",
    "    TestValueMeanError(),\n",
    "    TestValueMAPE(),\n",
    "    TestValueAbsMaxError(),\n",
    "    TestValueR2Score()\n",
    "])\n",
    "\n",
    "regression_performance_dataset_tests.run(reference_data=housing_ref, current_data=housing_cur)\n",
    "regression_performance_dataset_tests"
   ],
   "outputs": [],
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "id": "c98ce59f0d757ac0"
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Classification Performance Tests"
   ],
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "attachments": {},
   "id": "4b13cc3ac4f786d9"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#dataset-level tests\n",
    "classification_performance_dataset_tests = TestSuite(tests=[\n",
    "    TestAccuracyScore(),\n",
    "    TestPrecisionScore(),\n",
    "    TestRecallScore(),\n",
    "    TestF1Score(),\n",
    "    TestPrecisionByClass(label=0),\n",
    "    TestPrecisionByClass(label=1),\n",
    "    TestPrecisionByClass(label=2),\n",
    "    TestRecallByClass(label=0),\n",
    "    TestRecallByClass(label=1),\n",
    "    TestRecallByClass(label=2),\n",
    "    TestF1ByClass(label=0),\n",
    "    TestF1ByClass(label=1),\n",
    "    TestF1ByClass(label=2),\n",
    "])\n",
    "\n",
    "classification_performance_dataset_tests.run(reference_data=iris_ref, current_data=iris_cur)\n",
    "classification_performance_dataset_tests"
   ],
   "outputs": [],
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "id": "72a43ba972ba09a8"
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Probabilistic Classification Performance Tests"
   ],
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "attachments": {},
   "id": "2b014e81b8a0a318"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#dataset-level tests\n",
    "prob_classification_performance_dataset_tests = TestSuite(tests=[\n",
    "    TestAccuracyScore(),\n",
    "    TestPrecisionScore(),\n",
    "    TestRecallScore(),\n",
    "    TestF1Score(),\n",
    "    TestRocAuc(),\n",
    "    TestLogLoss(),\n",
    "    TestPrecisionByClass(label=0),\n",
    "    TestPrecisionByClass(label=1),\n",
    "    TestRecallByClass(label=0),\n",
    "    TestRecallByClass(label=1),\n",
    "    TestF1ByClass(label=0),\n",
    "    TestF1ByClass(label=1),\n",
    "\n",
    "])\n",
    "\n",
    "prob_classification_performance_dataset_tests.run(reference_data=bcancer_ref, current_data=bcancer_cur)\n",
    "prob_classification_performance_dataset_tests"
   ],
   "outputs": [],
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "id": "8c2f6281ba61ba57"
  },
  {
   "cell_type": "markdown",
   "source": [
    "## How to set test parameters?"
   ],
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "attachments": {},
   "id": "a75743b753fa5ac3"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#simple test parameters\n",
    "feature_level_tests = TestSuite(tests=[\n",
    "    TestMeanInNSigmas(column_name='hours-per-week', n_sigmas=3),\n",
    "    TestShareOfOutRangeValues(column_name='hours-per-week', lte=0),\n",
    "    TestColumnShareOfMissingValues(column_name='education', lt=0.2),\n",
    "])\n",
    "\n",
    "feature_level_tests.run(reference_data=adult_ref, current_data=adult_cur)\n",
    "feature_level_tests"
   ],
   "outputs": [],
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "id": "b66fab5f18798751"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#options\n",
    "color_scheme = ColorOptions(\n",
    "    primary_color=\"#5a86ad\",\n",
    "    fill_color=\"#fff4f2\",\n",
    "    zero_line_color=\"#016795\",\n",
    "    current_data_color= \"#c292a1\" ,\n",
    "    reference_data_color=\"#017b92\",\n",
    ")"
   ],
   "outputs": [],
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "id": "6b7877d93070b5e5"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "data_drift_column_tests = TestSuite(tests=[\n",
    "    TestColumnDrift(column_name='education-num', stattest='psi')\n",
    "], options=[color_scheme])\n",
    "\n",
    "data_drift_column_tests.run(reference_data=adult_ref, current_data=adult_cur)\n",
    "data_drift_column_tests"
   ],
   "outputs": [],
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "id": "dec4a403d8b68ac0"
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Support Evidently\n",
    "Did you find the example useful? Star Evidently on GitHub to contribute back! This helps us continue creating free open-source tools for the community. https://github.com/evidentlyai/evidently"
   ],
   "metadata": {},
   "id": "e8136c625eb1fdd5"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
